#define LOGGER_LEVEL LL_WARN

#include "RssReader.h"

#include "tinyxml2.h"
#include "../../include/simhash/Simhasher.hpp"


#include <iostream>
#include <regex>
#include <fstream>


using std::cout;
using std::endl;
using std::ofstream;

using namespace tinyxml2;
using namespace simhash;

RssReader::RssReader(): simhasher("../../include/simhash/dict/jieba.dict.utf8", 
                                  "../../include/simhash/dict/hmm_model.utf8", 
                                  "../../include/simhash/dict/idf.utf8", 
                                  "../../include/simhash/dict/stop_words.utf8") {}

void RssReader::parseRss (const vector<string> &fileName) {
    _rss.reserve(5120 * sizeof(RssItem));
    int id = 1;

    for(auto &elem : fileName) {
        string filename = string("./xml/") + elem;
        /* string filename = string("./xml/") + string("coolshell.xml"); */
        XMLDocument doc;
        doc.LoadFile(filename.c_str());
        if(doc.ErrorID()) {
            perror("LoadFile");
            return;
        }

        XMLElement *root = doc.FirstChildElement("rss");
        XMLElement *itemElement = root->FirstChildElement("channel")->FirstChildElement("item");

        /* int id = 1; */

        do {
            struct RssItem item;
            std::regex re("<.*?>");

            XMLElement *titleElement = itemElement->FirstChildElement("title");
            item.title = titleElement->GetText();

            XMLElement *linkElement = itemElement->FirstChildElement("link");
            item.link = linkElement->GetText();

            XMLElement *descriptionElment = itemElement->FirstChildElement("description");
            if(descriptionElment) {
                item.description = descriptionElment->GetText();
                item.description = std::regex_replace(item.description, re, string(""));
            }
            else {
                item.description = "";
            }

            XMLElement *contentElement = itemElement->FirstChildElement("content:encoded");
            if(contentElement) {
                /* cout << "有" << endl; */
                item.content = contentElement->GetText();
                item.content = std::regex_replace(item.content, re, string(""));
            }
            else {
                item.content = "";
            }

            item.id = std::to_string(id);

            string sNew = item.title + item.description + item.content;
            //cout << "------------" << endl;
            //cout << sNew << endl;
            bool repeat = isRepeat(sNew);//若重复，则返回true

            
            /* cout << (repeat ? 0 : id) << endl;//调试 */
            if(!repeat) {//没有重复才push进容器存起来
                _rss.push_back(item);
                ++id;    
            }

            //重复就跳过
        } while (itemElement = itemElement->NextSiblingElement("item"));
    }
}

bool RssReader::isRepeat(const string &sNew) {
    //Simhasher simhasher("../../include/simhash/dict/jieba.dict.utf8", 
    //                    "../../include/simhash/dict/hmm_model.utf8", 
    //                    "../../include/simhash/dict/idf.utf8", 
    //                    "../../include/simhash/dict/stop_words.utf8");
    size_t topN = 5;
    uint64_t simhashValue = 0;
    vector<std::pair<string, double>> res;
    simhasher.extract(sNew, res, topN);
    simhasher.make(sNew, topN, simhashValue);

    int sizeOld = _hash.size();
    _hash.insert(simhashValue);
    int sizeNew = _hash.size();

    return sizeOld == sizeNew;

    //for(int i = 0; i < _simhash.size(); ++i) {
    //    uint64_t oldSimhashValue = _simhash[i];
    //    bool equal = Simhasher::isEqual(oldSimhashValue, simhashValue);
    //    if(equal) {
    //        return true;
    //    }
    //}
    //_simhash.push_back(simhashValue); 
    //return false;



    //for(int i = 0; i < _rss.size(); ++i) {
    //    string sOld = _rss[i].title + _rss[i].description + _rss[i].content;
    //    Simhasher simhasher("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8");
    //    size_t topN = 5;
    //    uint64_t u1 = 0;
    //    uint64_t u2 = 0;
    //    vector<std::pair<string, double>> res1;
    //    vector<std::pair<string, double>> res2;
    //    simhasher.extract(sOld, res1, topN);
    //    simhasher.make(sOld, topN, u1);
    //    simhasher.extract(sNew, res2, topN);
    //    simhasher.make(sNew, topN, u2);

    //    bool equal = Simhasher::isEqual(u1, u2);
    //    if(equal) {
    //        return equal;
    //    }
    //}
    //return false;
}

void RssReader::dumpWebLib (const string &outputFile) {
    ofstream ofs(outputFile, std::ios::app);
    if (!ofs) {
        perror("RssReader.cc dumpWebLib");
        return;
    }
    for(int i = 0; i < _rss.size(); ++i){
        //ofs << "<doc>" << endl;
        //ofs << "   <docid>" << std::to_string(i + 1) << "</docid>";
        //ofs << "\n   <title>";
        //ofs << _rss[i].title << "</title>";
        //ofs << "\n   <link>";
        //ofs << _rss[i].link << "</link>";
        //ofs << "\n   <description>";
        //ofs << _rss[i].description << "</description>";
        ///* ofs << "\n   <content>"; */
        ///* ofs << _rss[i].content << "</content>"; */
        //ofs << "\n</doc>" << endl;

        /* ofs << _rss[i].title << _rss[i].link << _rss[i].description << _rss[i].id << endl; */

        ofs << "<doc>" << endl;//6
        ofs << "    <docid>" << _rss[i].id << "</docid>" << endl;//20 + 数字长度
        ofs << "    <title>" <<_rss[i].title << "</title>" << endl;//20
        ofs << "    <link>" << _rss[i].link << "</link>" << endl;//18
        ofs << "    <description>" << _rss[i].description << "</description>" << endl;//32
        ofs << "    <content>" << _rss[i].content << "</content>" << endl;//24
    }
    ofs.close();
}

void RssReader::dumpOffsetLib(const string &outputFile) {
    ofstream ofs(outputFile, std::ios::app);
    if(!ofs) {
        perror("RssReader.cc dumpOffsetLib");
        return;
    }
    int begin = 0;
    for(size_t i = 0; i < _rss.size(); ++i) {
        size_t length = getLength(_rss[i]) + 120;
        ofs << std::to_string(i + 1) << "   ";
        ofs << begin << "   "; 
        ofs << length << "   " << endl;
        begin += length;
    }
    ofs.close();
}


size_t RssReader::getLength(const RssItem &item) {
    return item.title.size() + item.link.size() + item.description.size() + item.content.size() + item.id.size();
}

